{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Collecting kfp\n",
      "  Downloading kfp-0.5.1.tar.gz (119 kB)\n",
      "\u001b[K     |████████████████████████████████| 119 kB 3.5 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: PyYAML in /usr/local/lib/python3.6/dist-packages (from kfp) (5.3)\n",
      "Requirement already satisfied: google-cloud-storage>=1.13.0 in /usr/local/lib/python3.6/dist-packages (from kfp) (1.25.0)\n",
      "Requirement already satisfied: kubernetes<12.0.0,>=8.0.0 in /usr/local/lib/python3.6/dist-packages (from kfp) (10.0.1)\n",
      "Requirement already satisfied: google-auth>=1.6.1 in /usr/local/lib/python3.6/dist-packages (from kfp) (1.11.0)\n",
      "Collecting requests_toolbelt>=0.8.0\n",
      "  Downloading requests_toolbelt-0.9.1-py2.py3-none-any.whl (54 kB)\n",
      "\u001b[K     |████████████████████████████████| 54 kB 4.0 MB/s  eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: cloudpickle in /usr/local/lib/python3.6/dist-packages (from kfp) (1.2.2)\n",
      "Collecting kfp-server-api<0.6.0,>=0.2.5\n",
      "  Downloading kfp-server-api-0.5.0.tar.gz (39 kB)\n",
      "Requirement already satisfied: jsonschema>=3.0.1 in /usr/local/lib/python3.6/dist-packages (from kfp) (3.2.0)\n",
      "Collecting tabulate\n",
      "  Downloading tabulate-0.8.7-py3-none-any.whl (24 kB)\n",
      "Collecting click\n",
      "  Downloading click-7.1.2-py2.py3-none-any.whl (82 kB)\n",
      "\u001b[K     |████████████████████████████████| 82 kB 1.5 MB/s  eta 0:00:01\n",
      "\u001b[?25hCollecting Deprecated\n",
      "  Downloading Deprecated-1.2.9-py2.py3-none-any.whl (8.6 kB)\n",
      "Collecting strip-hints\n",
      "  Downloading strip-hints-0.1.9.tar.gz (30 kB)\n",
      "Requirement already satisfied: google-cloud-core<2.0dev,>=1.2.0 in /usr/local/lib/python3.6/dist-packages (from google-cloud-storage>=1.13.0->kfp) (1.3.0)\n",
      "Requirement already satisfied: google-resumable-media<0.6dev,>=0.5.0 in /usr/local/lib/python3.6/dist-packages (from google-cloud-storage>=1.13.0->kfp) (0.5.0)\n",
      "Requirement already satisfied: six>=1.9.0 in /usr/lib/python3/dist-packages (from kubernetes<12.0.0,>=8.0.0->kfp) (1.11.0)\n",
      "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /usr/local/lib/python3.6/dist-packages (from kubernetes<12.0.0,>=8.0.0->kfp) (0.57.0)\n",
      "Requirement already satisfied: python-dateutil>=2.5.3 in /usr/local/lib/python3.6/dist-packages (from kubernetes<12.0.0,>=8.0.0->kfp) (2.8.1)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from kubernetes<12.0.0,>=8.0.0->kfp) (2.22.0)\n",
      "Requirement already satisfied: requests-oauthlib in /usr/local/lib/python3.6/dist-packages (from kubernetes<12.0.0,>=8.0.0->kfp) (1.3.0)\n",
      "Requirement already satisfied: setuptools>=21.0.0 in /usr/local/lib/python3.6/dist-packages (from kubernetes<12.0.0,>=8.0.0->kfp) (45.1.0)\n",
      "Requirement already satisfied: urllib3>=1.24.2 in /usr/local/lib/python3.6/dist-packages (from kubernetes<12.0.0,>=8.0.0->kfp) (1.25.8)\n",
      "Requirement already satisfied: certifi>=14.05.14 in /usr/local/lib/python3.6/dist-packages (from kubernetes<12.0.0,>=8.0.0->kfp) (2019.11.28)\n",
      "Requirement already satisfied: rsa<4.1,>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.6.1->kfp) (4.0)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.6.1->kfp) (4.0.0)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.6.1->kfp) (0.2.8)\n",
      "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.6/dist-packages (from jsonschema>=3.0.1->kfp) (19.3.0)\n",
      "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from jsonschema>=3.0.1->kfp) (1.4.0)\n",
      "Requirement already satisfied: pyrsistent>=0.14.0 in /usr/local/lib/python3.6/dist-packages (from jsonschema>=3.0.1->kfp) (0.15.7)\n",
      "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.6/dist-packages (from Deprecated->kfp) (1.11.2)\n",
      "Requirement already satisfied: wheel in /usr/lib/python3/dist-packages (from strip-hints->kfp) (0.30.0)\n",
      "Requirement already satisfied: google-api-core<2.0.0dev,>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp) (1.16.0)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /usr/lib/python3/dist-packages (from requests->kubernetes<12.0.0,>=8.0.0->kfp) (2.6)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->kubernetes<12.0.0,>=8.0.0->kfp) (3.0.4)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib->kubernetes<12.0.0,>=8.0.0->kfp) (3.1.0)\n",
      "Requirement already satisfied: pyasn1>=0.1.3 in /usr/local/lib/python3.6/dist-packages (from rsa<4.1,>=3.1.4->google-auth>=1.6.1->kfp) (0.4.8)\n",
      "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->jsonschema>=3.0.1->kfp) (2.1.0)\n",
      "Requirement already satisfied: protobuf>=3.4.0 in /usr/local/lib/python3.6/dist-packages (from google-api-core<2.0.0dev,>=1.16.0->google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp) (3.11.2)\n",
      "Requirement already satisfied: pytz in /usr/local/lib/python3.6/dist-packages (from google-api-core<2.0.0dev,>=1.16.0->google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp) (2019.3)\n",
      "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.6.0 in /usr/local/lib/python3.6/dist-packages (from google-api-core<2.0.0dev,>=1.16.0->google-cloud-core<2.0dev,>=1.2.0->google-cloud-storage>=1.13.0->kfp) (1.51.0)\n",
      "Building wheels for collected packages: kfp, kfp-server-api, strip-hints\n",
      "  Building wheel for kfp (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for kfp: filename=kfp-0.5.1-py3-none-any.whl size=163151 sha256=da5b540ae9834d37659146f0576997ffd8f7a7e2b305e1eb7b2a99dd4745930b\n",
      "  Stored in directory: /home/jovyan/.cache/pip/wheels/2f/26/f9/e3836cb6e6cabd63ef912304e18a852ac29cb870a4a0b85f98\n",
      "  Building wheel for kfp-server-api (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for kfp-server-api: filename=kfp_server_api-0.5.0-py3-none-any.whl size=106319 sha256=84f55948cc254c0f836dffdfd51574a828ae8a503a2ca9198acf7a27ca2aaea7\n",
      "  Stored in directory: /home/jovyan/.cache/pip/wheels/73/36/4e/bfe2efeeea4f74f04984ebe1d44136202b72191302f4760951\n",
      "  Building wheel for strip-hints (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for strip-hints: filename=strip_hints-0.1.9-py2.py3-none-any.whl size=24671 sha256=3bcfd573a91f5f4c46d23509ac3fee9a0cf351b414e00ed505a8f71d0e6a1141\n",
      "  Stored in directory: /home/jovyan/.cache/pip/wheels/21/6d/fa/7ed7c0560e1ef39ebabd5cc0241e7fca711660bae1ad752e2b\n",
      "Successfully built kfp kfp-server-api strip-hints\n",
      "Installing collected packages: requests-toolbelt, kfp-server-api, tabulate, click, Deprecated, strip-hints, kfp\n",
      "\u001b[33m  WARNING: The script tabulate is installed in '/home/jovyan/.local/bin' which is not on PATH.\n",
      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n",
      "\u001b[33m  WARNING: The script strip-hints is installed in '/home/jovyan/.local/bin' which is not on PATH.\n",
      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n",
      "\u001b[33m  WARNING: The scripts dsl-compile and kfp are installed in '/home/jovyan/.local/bin' which is not on PATH.\n",
      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n",
      "Successfully installed Deprecated-1.2.9 click-7.1.2 kfp-0.5.1 kfp-server-api-0.5.0 requests-toolbelt-0.9.1 strip-hints-0.1.9 tabulate-0.8.7\n",
      "\u001b[33mWARNING: You are using pip version 20.0.2; however, version 20.1 is available.\n",
      "You should consider upgrading via the '/usr/bin/python3 -m pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip3 install kfp\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import kfp\n",
    "import kubernetes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "container_manifest = {\n",
    "    \"apiVersion\": \"sparkoperator.k8s.io/v1beta2\",\n",
    "    \"kind\": \"SparkApplication\",\n",
    "    \"metadata\": {\n",
    "        \"name\": \"spark-app\",\n",
    "        \"namespace\": \"kubeflow\"\n",
    "    },\n",
    "    \"spec\": {\n",
    "        \"type\": \"Scala\",\n",
    "        \"mode\": \"cluster\",\n",
    "        \"image\": \"docker.io/rawkintrevo/covid-basis-vectors:0.2.0\",\n",
    "        \"imagePullPolicy\": \"Always\",\n",
    "        \"hadoopConf\": {\n",
    "            \"fs.gs.project.id\": \"kubeflow-hacky-hacky\",\n",
    "            \"fs.gs.system.bucket\": \"covid-dicoms\",\n",
    "            \"fs.gs.impl\" : \"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem\",\n",
    "            \"google.cloud.auth.service.account.enable\": \"true\",\n",
    "            \"google.cloud.auth.service.account.json.keyfile\": \"/mnt/secrets/user-gcp-sa.json\",\n",
    "        },\n",
    "        \"mainClass\": \"org.rawkintrevo.covid.App\",\n",
    "        \"mainApplicationFile\": \"local:///covid-0.1-jar-with-dependencies.jar\", # See the Dockerfile\n",
    "        \"arguments\": [\"245\", \"15\", \"1\"],\n",
    "        \"sparkVersion\": \"2.4.5\",\n",
    "        \"restartPolicy\": {\n",
    "            \"type\": \"Never\"\n",
    "        },\n",
    "        \"driver\": {\n",
    "            \"cores\": 1,\n",
    "            \"secrets\": [\n",
    "                {\"name\": \"user-gcp-sa\",\n",
    "                 \"path\": \"/mnt/secrets\",\n",
    "                 \"secretType\": \"GCPServiceAccount\"\n",
    "                 }\n",
    "            ],\n",
    "\n",
    "            \"coreLimit\": \"1200m\",\n",
    "            \"memory\": \"512m\",\n",
    "            \"labels\": {\n",
    "                \"version\": \"2.4.5\",\n",
    "            },\n",
    "            \"serviceAccount\": \"spark-operatoroperator-sa\", # also try spark-operatoroperator-sa\n",
    "        },\n",
    "        \"executor\": {\n",
    "            \"cores\": 1,\n",
    "            \"secrets\": [\n",
    "                {\"name\": \"user-gcp-sa\",\n",
    "                 \"path\": \"/mnt/secrets\",\n",
    "                 \"secretType\": \"GCPServiceAccount\"\n",
    "                 }\n",
    "            ],\n",
    "            \"instances\": 4,\n",
    "            \"memory\": \"4084m\"\n",
    "        },\n",
    "        \"labels\": {\n",
    "            \"version\": \"2.4.5\"\n",
    "        },\n",
    "\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from kfp.gcp import use_gcp_secret\n",
    "@kfp.dsl.pipeline(\n",
    "    name=\"Covid DICOM Pipe v2\",\n",
    "    description=\"Create Basis Vectors for Lung Images\"\n",
    ")\n",
    "def covid_dicom_pipeline():\n",
    "    vop = kfp.dsl.VolumeOp(\n",
    "        name=\"requisition-PVC\",\n",
    "        resource_name=\"datapvc\",\n",
    "        size=\"20Gi\", #10 Gi blows up...\n",
    "        modes=kfp.dsl.VOLUME_MODE_RWO\n",
    "    )\n",
    "    step1 = kfp.dsl.ContainerOp(\n",
    "        name=\"download-dicom\",\n",
    "        image=\"rawkintrevo/download-dicom:0.0.0.4\",\n",
    "        command=[\"/run.sh\"],\n",
    "        pvolumes={\"/data\": vop.volume}\n",
    "    )\n",
    "    step2 = kfp.dsl.ContainerOp(\n",
    "        name=\"convert-dicoms-to-vectors\",\n",
    "        image=\"rawkintrevo/covid-prep-dicom:0.9.5\",\n",
    "        arguments=[\n",
    "            '--bucket_name', \"covid-dicoms\",\n",
    "        ],\n",
    "        command=[\"python\", \"/program.py\"],\n",
    "        pvolumes={\"/mnt/data\": step1.pvolume}\n",
    "    ).apply(kfp.gcp.use_gcp_secret(secret_name='user-gcp-sa'))\n",
    "    rop = kfp.dsl.ResourceOp(\n",
    "        name=\"calculate-basis-vectors\",\n",
    "        k8s_resource=container_manifest,\n",
    "        action=\"create\",\n",
    "        success_condition=\"status.applicationState.state == COMPLETED\"\n",
    "    ).after(step2)\n",
    "    pyviz = kfp.dsl.ContainerOp(\n",
    "        name=\"visualize-slice-of-dicom\",\n",
    "        image=\"rawkintrevo/visualize-dicom-output:0.0.11\",\n",
    "        command=[\"python\", \"/program.py\"],\n",
    "        arguments=[\n",
    "            '--bucket_name', \"covid-dicoms\",\n",
    "        ],\n",
    "    ).apply(kfp.gcp.use_gcp_secret(secret_name='user-gcp-sa')).after(rop)\n",
    "    \n",
    "\n",
    "kfp.compiler.Compiler().compile(covid_dicom_pipeline,\"dicom-pipeline-2.zip\")\n",
    "client = kfp.Client()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Experiment link <a href=\"/pipeline/#/experiments/details/a7292089-5186-4e53-b0bb-9264dfbb9775\" target=\"_blank\" >here</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run link <a href=\"/pipeline/#/runs/details/0f3f3d01-f6c4-4216-8e03-396c49fa040f\" target=\"_blank\" >here</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "my_experiment = client.create_experiment(name='my-experiments')\n",
    "my_run = client.run_pipeline(my_experiment.id, 'my-run1', 'dicom-pipeline-2.zip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
